from lxml import etree
from urllib.request import *

url = 'https://www.zhipin.com/job_detail/0281f879af37f3231XV63925FVs~.html'


def get_web_content(url, refer):
	user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36'
	headers = {'User-Agent': user_agent, 'Connection': 'keep-alive', 'Referer': refer}
	request = Request(url, headers=headers)
	print('request: ', url)
	# print('start request time -->', datetime.datetime.now())
	with urlopen(request) as result:
		data = result.read()
		dom = etree.HTML(data)
		block_title = dom.xpath('./div[@class="text"]')
		print(type(block_title))
		for inner in block_title:
			print(inner)




get_web_content(url, 'https://www.zhipin.com/c101010100-p100101/')
